package com.jahentao.file.large;

import com.google.gson.Gson;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 数据处理应用
 * 提取感兴趣的字段
 * Id
 * PostTypeId (PostTypeId一共有两种，一种是question，一种是answer)
 * CreateDate
 * Title
 * Body
 * Score
 * ViewCount
 * Tags (question post会有Tags，answer post没有Tags)
 * URLs (需要从每个post的body当中，提取出URLs)
 * AnswerCount
 * CommentCount
 * FavoriteCount
 */
public class App {

    public static void main(String[] args) {
        //
        int count=0;
        File theFile = new File("H:\\stackoverflow.com-Posts\\Posts.xml");
        LineIterator it = null;
        RowElement row = null;
        Set<String> interestedFieldSet = new HashSet<String>(11) {{
            add("Id"); add("PostTypeId"); add("CreateDate"); add("Title"); add("Body"); add("Score");
            add("ViewCount"); add("Tags"); add("AnswerCount"); add("CommentCount"); add("FavoriteCount");
        }};
        Map<String, String> map = new HashMap<String, String>(12);

        Pattern  p = Pattern.compile("&lt;a\\s+href\\s*=\\s*&quot;\\S*&quot;");

        Gson gson = new Gson();

        // 连接到 mongodb 服务
        MongoClient mongoClient = new MongoClient( "localhost" , 27017 );
        // 连接到数据库
        MongoDatabase mongoDatabase = mongoClient.getDatabase("posts");
        MongoCollection<org.bson.Document> collection = mongoDatabase.getCollection("row");

        try {
            it = FileUtils.lineIterator(theFile, "UTF-8");
            while (it.hasNext() && count<10000000) {
                String line = it.nextLine().trim();
//                System.out.println("Line "+count+": "+line);
                // 算了，自己写个提取属性的解析器吧
                if (row == null) {
                    row = new RowElement(line);
                } else {
                    row.parseAttributes(line);
                }

                for (int i=0; i<row.getAttributes().getLength(); i++) {
                    Node attr = row.getAttributes().item(i);
                    String fileName = attr.getNodeName();
                    if (interestedFieldSet.contains(fileName)) {

                        if ("Body".equals(fileName)) {
                            String value = attr.getNodeValue();
                            // 正则表达式匹配，找出引用
                            Matcher m = p.matcher(value);

                            List<String> urls = new ArrayList<String>();

                            while(m.find()) {
                                String a = m.group();
                                String[] split = a.split("&lt;a\\s+href\\s*=\\s*&quot;");
                                String h = split[1].substring(0, split[1].length() - 6);
//                                System.out.println(h);
                                urls.add(h);
                            }

                            if (urls.size() > 0) {
                                map.put("URLs", gson.toJson(urls));
//                                System.out.println(gson.toJson(urls));
                            }
                        }

                        // 提取感兴趣的属性
                        map.put(fileName, attr.getNodeValue());
                    }
                }

//                System.out.println(map);
                // 存储数据库
                org.bson.Document document = new org.bson.Document();
                Set<String> fieldNameSet = map.keySet();
                Iterator<String> iterator = fieldNameSet.iterator();
                while (iterator.hasNext()) {
                    String fieldName = iterator.next();
                    document.append(fieldName, map.get(fieldName));
                }
//                System.out.println(document);
                collection.insertOne(document);

                count ++;
                System.out.println(count);
                // 一定要清除
                row.clearAttributes();
                map.clear();
            }

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            LineIterator.closeQuietly(it);
        }
    }
}
